package com.github.rabbithome.processor;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

import java.util.List;

/**
 * Created by tomalloc on 17-1-6.
 */
public class WubaPageProcessor implements PageProcessor {
    private Site site;



    public WubaPageProcessor(Site site) {
        this.site = site;
    }

    @Override
    public void process(Page page) {
        if(page.getResultItems().isSkip()){
            return;
        }
        List<Selectable> selectableList = page.getHtml().css("ul.listUl>li[logr]").nodes();
        page.putField("data",selectableList);
        String href=page.getHtml().css("div.pager>a.next","href").get();
        page.addTargetRequest(href);
    }

    @Override
    public Site getSite() {
        return site;
    }

}
